package com.guiji.quartz.domain;

import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.config.ProxyConfig;
import com.guiji.quartz.task.DataSaveUtil;
import com.guiji.quartz.util.ChromeDriverClonable;
import com.guiji.quartz.util.DelPicUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.*;
import org.openqa.selenium.chrome.ChromeOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: wangxw
 * @date 2021/12/3 15:10
 */
public class BizjournalsBank {
	private static final Logger log = LoggerFactory.getLogger(BizjournalsBank.class);

	static {
		String chromedriver = "/usr/bin/chromedriver";
		String chrome = "/opt/google/chrome/chrome";

		String osname = System.getProperty("os.name");
		if (osname.contains("Windows")) {
			System.setProperty("webdriver.chrome.driver", "E:\\plugins\\chromedriver.exe");
			System.setProperty("webdriver.chrome.bin", "C:\\Users\\Administrator\\AppData\\Local\\google\\Chrome\\Application\\chrome.exe");
			System.setProperty("webdriver.chrome.marionette", "C:\\Users\\Administrator\\AppData\\Local\\google\\Chrome\\Application\\chrome.exe");
		} else {
			log.info("driver初始化");
			System.setProperty("webdriver.chrome.driver", chromedriver);
			System.setProperty("webdriver.chrome.bin", chrome);
			System.setProperty("webdriver.chrome.marionette", chrome);

		}
	}

	public static void mainMethod() throws IOException, InterruptedException {
		log.info("BizjournalsBank的driver开启");
		ChromeDriverClonable driver = null;
		try {
			driver = null;
			ChromeOptions chromeOptions = new ChromeOptions();
			chromeOptions.addArguments("--headless");
			chromeOptions.addArguments("--no-sandbox");
			chromeOptions.addArguments("window-size=13660x768");
			chromeOptions.addArguments("--disable-gpu");
			driver = new ChromeDriverClonable(chromeOptions);
			driver.get("https://www.bizjournals.com/news/banking-and-financial-services");
			String doc = driver.getPageSource();
			if (null != doc && doc.length() > 0) {
				Set<Cookie> selenCookies = driver.manage().getCookies();
				List<HttpCookie> cookies = transCookies(selenCookies);
				Set<String> resUrl = getUrl(doc);
				getContent(resUrl, cookies);
			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (null != driver) {
				driver.quit();
				log.info("BizjournalsBank的driver关闭");
			}
		}
	}


	private static List<HttpCookie> transCookies(Set<Cookie> srcCookies) {
		List<HttpCookie> destCookies = new ArrayList<>();

		for (Cookie srcCookie : srcCookies) {
			HttpCookie destCookie = new HttpCookie(srcCookie.getName(), srcCookie.getValue());
			destCookies.add(destCookie);
		}

		return destCookies;

	}

	private static void getContent(Set<String> listContent, List<HttpCookie> cookies) throws MalformedURLException, URISyntaxException, InterruptedException {
		for (String url : listContent) {
			Thread.sleep(1000);
			if (!url.startsWith("https")) {
				url = new StringBuffer("https://www.bizjournals.com/").append(url).toString();
			}
			if (null != cookies && cookies.size() > 0) {
				CookieManager ccm = new CookieManager();
				CookieStore CookieStore = ccm.getCookieStore();
				for (HttpCookie cookie : cookies) {
					URL domain = new URL("https://www.bizjournals.com");
					CookieStore.add(domain.toURI(), cookie);
				}
				HttpRequest.setCookieManager(ccm);
			}

			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc;
			JXDocument jxDocument;
			try {
				HttpResponse response = HttpRequest.get(url)
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
					.header("Refer", "https://www.bizjournals.com/news/banking-and-financial-services")
					.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
					.setConnectionTimeout(30000).execute();
				doc = response.body();
				jxDocument = new JXDocument(doc);
			} catch (Exception e) {
				log.info("代理失败2" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;
			}
			StringBuffer content = new StringBuffer();
			String contentStr = null;
			try {

				Document document = Jsoup.parse(doc);
				Element mainArea = document.getElementsByClass("content cXenseParse").get(0);
				Elements pElements = mainArea.select("P");
				if (null != pElements && pElements.size() > 0) {
					for (Element pdom : pElements) {
						String temp = pdom.text();
						if (null != temp && !temp.toString().contains("Sign up here") && !temp.toString().contains("Only the headline and picture of this report")) {
							content.append(temp + "$$$");
							pdom.remove();
						}
					}
				}
				contentStr = content.toString();
			} catch (Exception e) {
				continue;
			}

			List<Object> srcImgs = new ArrayList<>();
			List<String> newImgs = new ArrayList<>();
			String img = null;
			try {
				String imgXpath = "//meta[@property='og:image']/@content";
				int imgSize = jxDocument.sel(imgXpath).size();
				if (imgSize > 0) {
					String temp = null;
					for (int k = 0; k < imgSize; k++) {
						temp = jxDocument.sel(imgXpath).get(k).toString();
						img = temp.strip();
						srcImgs.add(img);
						break;
					}
				}

			} catch (Exception e) {
				log.error("可能没有图片");
				img = "";
			} finally {
				if (srcImgs.size() > 0) {
					newImgs = DelPicUtil.delPic(srcImgs);
				}
			}

			Date date = new Date();
			if (contentStr.length() > 50000 || contentStr.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				System.out.println("=======================================================");
				System.out.println(url);
				System.out.println(title);

				for (Object imgitem : srcImgs) {
					System.out.println(" " + imgitem);
				}
				System.out.println(contentStr);
				System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
				DataSaveUtil.saveData(title, contentStr, newImgs, url, date, "https://www.bizjournals.com/news/banking-and-financial-services", "Bizjournals-Banking");
				log.info(Thread.currentThread().getName() + "-----------------Bizjournals-Banking ----------------");
			} catch (Exception e) {
				continue;
			}

		}
	}


	private static Set<String> getUrl(String content) {

		//bizjournals/news/2021/11/15/inflation-labor-small-business-score-covid.html
		Set<String> res = new HashSet<>();
		String pattern = "bizjournals/news/[0-9]{4}/[0-9]{1,2}/[0-9]{1,2}/.*?html";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			res.add(m.group());
		}
		return res;
	}

//	public static void main(String[] args) throws Exception {
//		mainMethod();
//	}

}



